package fetcher

import (
	"net/http"
	"fmt"
	"golang.org/x/text/transform"
	"io/ioutil"
	"bufio"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"log"
	"time"
	"gxgeek.com/gxgeek/learngo/crawler/config"
)

var (
	rateLimiter = time.Tick(time.Second/config.Qps)
	verboseLogging = false
)
func SetVerboseLogging() {
	verboseLogging = true
}

func Fetcher(url string) ([]byte, error) {
	<- rateLimiter
	if verboseLogging {
		log.Printf("Fetching url %s", url)
	}
	request, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}
	request.Header.Add("User-Agent","Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Mobile Safari/537.36")
	cilent := http.Client{}
	resp, err := cilent.Do(request)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()


	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		errorf := fmt.Errorf("wrong bad body : %d", resp.StatusCode)
		return nil, errorf
	}
	bodyReader := bufio.NewReader(resp.Body)
	determineEncoding := determineEncoding(bodyReader)
	reader := transform.NewReader(bodyReader, determineEncoding.NewDecoder())
	body, err := ioutil.ReadAll(reader)
	return body, err

}
func determineEncoding(r *bufio.Reader) encoding.Encoding {
	bytes, err := r.Peek(1024)
	if err != nil {
		log.Panicf("Fetcher error :%v ", err)
		return unicode.UTF8
	}
	e, _, _ := charset.DetermineEncoding(bytes, "")
	return e

}
